# -*- coding:UTF-8 -*-
# 开发人员: limenghui
# 开发时间：2024-09-21 下午11:25
# 文件名称：
# 开发工具：pycharm


from time import sleep
import requests
import io
import json
import csv
import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
    'Content-Type': 'application/json;charset=utf-8'
}


options = webdriver.EdgeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
# 屏蔽密码提示框
# prefs = {'credentials_enable_service':False,'profile.password_manager_enabled':False}
# 反爬虫特征处理
options.add_argument('disable-blink-features=AutomationControlled')

options.add_experimental_option("prefs", {
    "download.default_directory": r"C:\Users\limenghui\PycharmProjects\CITIC_BANK\WORK",
    "download.prompt_for_download": False,
    "plugins.always_open_plugin_dir": True,
    "profile.default_content_settings.popups": 0,
})
options.add_argument("--headless")

starttime = time.time()

with open("fz_gk_url.json",'r', encoding='UTF-8') as f:
     fz_gk_url = json.load(f)
print(len(fz_gk_url))

with open('gk_data.csv', 'a', newline='') as f:
    writer = csv.writer(f)
    writer.writerow(['pubfz','pubcurrurl', 'pubtitle', 'pubtime', 'pubcontent', 'pubhtml','pdf_url'])  # 写入表头
    try:
        ct = 0
        for fz,urls in fz_gk_url.items():
            gk_dict  = {}
            for url in urls:
                ct += 1
                driver = webdriver.Edge(options=options)
                driver.get(url)
                driver.implicitly_wait(5)
                pubcurrurl = driver.current_url
                pubtitle = driver.find_element(By.XPATH, '//h2').text
                pubtime = driver.find_element(By.XPATH, '//*[@id="shijian"]').text
                pubcontent = driver.find_element(By.XPATH, '// *[ @ id = "zoom"]').text
                pubhtml = driver.page_source  # 获取页面源代码
                try:
                    pubpdf_url =  driver.find_element(By.XPATH,'//*[@id="zoom"]/p//a')
                    pdf_url = pubpdf_url.get_attribute('href')
                except Exception as e:
                    pdf_url = None
                gk_dict['pubfz'] = fz
                gk_dict['pubcurrurl'] = pubcurrurl
                gk_dict['pubtitle'] = pubtitle
                gk_dict['pubcontent'] = pubcontent
                gk_dict['pubhtml'] = pubhtml
                gk_dict['pdf_url'] = pdf_url
                writer.writerow([fz,pubcurrurl, pubtitle, pubtime, pubcontent, pubhtml,pdf_url])
                print(pubtitle,pubtime,pdf_url,sep='|')
                driver.close()
    except Exception as e:
        pass
endtime = time.time()
spendtime = endtime - starttime
print("="*50)
print(f'共获得条{ct}数据')
print(f"共运行{spendtime}秒，大概耗时{spendtime/60}分钟。")
